{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "以下代码，我没有在TinyMind上面跑，我TinyMind的Notebooks死机了。我先解锁后面课程，然后抽空再跑这个。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def swish(x):\n",
    "  return x*tf.nn.sigmoid(x) #sigmoid(x):1/(1+e^-x)\n",
    "  \n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0.0,x,alpha*tf.nn.elu(x))\n",
    "    \n",
    "#selu(x):y=lamdba*(case when x>0 then x else when x<=0 \n",
    "    #then alpha*(e^x-1)) 其效果比Batch Normalization要好。\n",
    "    \n",
    "def relu(x):\n",
    "  return tf.nn.relu(x)\n",
    "  \n",
    "def activation(x):\n",
    "  #\n",
    "  return swish(x)  #使用swish(x)函数的效果是最好的！\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "###X_train,Y_train,X_test,Y_test=mnist.train.images,mnist.train.labels,mnist.test.images,mnist.test.labels\n",
    "x = tf.placeholder(tf.float32, [None, 28,28,1]) ###定义了一个二维张量\n",
    "y=tf.placeholder('float',[None,10])\n",
    "tf.shape(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "28×28是图像的长宽像素值，1表示图像的通道数(这里是黑白图像所以为1通道)。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#exponetial lr decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000,tf.shape(x)[0]))  ###设置epoch_steps  #tf.div除法\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch=global_step//epoch_steps      \n",
    "decay_times=current_epoch\n",
    "current_learning_rate = tf.multiply(init_learning_rate,\n",
    "                    tf.pow(0.575,tf.to_float(decay_times))) #0.575的decay_times次方"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "初始化权重与定义网络。我将搭建一个拥有3个卷基层和三个池化层，随后接1个全连接层和1个输出层的卷积神经网络。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义权重函数\n",
    "def init_weights(shape):\n",
    "    return(tf.Variable(tf.truncated_normal(shape,stddev=0.01)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#初始化权重\n",
    "w=init_weights([3,3,1,32])#patch大小为3*3，输入维度1，输出维度32\n",
    "w2=init_weights([3,3,32,64])#patch大小为3*3，输入维度32，输出维度64\n",
    "w3=init_weights([3,3,64,128])#patch大小为3*3，输入维度64，输出维度128\n",
    "w4=init_weights([128*4*4,625])#全连接层，输入128*4*4，是上一层输出数据三维转变为1维，输出维度625\n",
    "w_o=init_weights([625,10])#输出层，输入维度625，输出维度10(表示10类)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义神经网络\n",
    "def model(X,w,w2,w3,w4,w_o,p_keep_conv,p_keep_hidden):\n",
    "    #定义第一组卷积层和池化层，最后dropout掉一些神经元\n",
    "    l1a=tf.nn.relu(tf.nn.conv2d(X,w,strides=[1,1,1,1],padding='SAME'))\n",
    "    l1=tf.nn.max_pool(l1a,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')\n",
    "    l1=tf.nn.dropout(l1,p_keep_conv)\n",
    "    \n",
    "    #定义第二组卷积层和池化层，最后dropout掉一些神经元\n",
    "    l2a=tf.nn.relu(tf.nn.conv2d(l1,w2,strides=[1,1,1,1],padding='SAME'))\n",
    "    l2=tf.nn.max_pool(l2a,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')\n",
    "    l2=tf.nn.dropout(l2,p_keep_conv)\n",
    "    \n",
    "    #定义第三组卷积层和池化层，最后dropout掉一些神经元\n",
    "    l3a=tf.nn.relu(tf.nn.conv2d(l2,w3,strides=[1,1,1,1],padding='SAME'))\n",
    "    l3=tf.nn.max_pool(l3a,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')\n",
    "    l3=tf.reshape(l3,[-1,w4.get_shape().as_list()[0]])\n",
    "    l3=tf.nn.dropout(l3,p_keep_conv)\n",
    "    \n",
    "    #全连接层，最后dropout掉一些神经元\n",
    "    b_1 = tf.Variable(tf.constant(0.001,shape=[625])) \n",
    "    l4=tf.nn.relu(tf.matmul(l3,w4)+b_1)###激活\n",
    "    l4=tf.nn.dropout(l4,p_keep_hidden)\n",
    "    \n",
    "    #输出层\n",
    "    b_2=tf.Variable(tf.constant(0.001,shape=[10]))\n",
    "    logits_x=tf.matmul(l4,w_o) + b_2\n",
    "    \n",
    "    return(logits_x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来定义dropout的占位符——keep_conv和keep_hidden，\n",
    "它表示一层中有多少比例的神经元被留下来，这个东西可以有效的避免过拟合问题。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p_keep_conv=tf.placeholder('float')\n",
    "p_keep_hidden=tf.placeholder('float')\n",
    "logits_x=model(x,w,w2,w3,w4,w_o,1.0,1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits_x))\n",
    "    \n",
    "l2_loss=tf.nn.l2_loss(w)+tf.nn.l2_loss(w2)+tf.nn.l2_loss(w3)+tf.nn.l2_loss(w4)+tf.nn.l2_loss(w_o)\n",
    "total_loss = cross_entropy +4e-5*l2_loss\n",
    "\n",
    "optimizer=tf.train.AdamOptimizer(current_learning_rate)\n",
    "gradients=optimizer.compute_gradients(total_loss)\n",
    "train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "  current_learning_rate).minimize(\n",
    "  total_loss,global_step=global_step)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练模型和评估模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for step in range(3000):\n",
    "    batch_xs,batch_ys = mnist.train.next_batch(100)\n",
    "    lr=1e-2\n",
    "    _,loss,l2_loss_value,total_loss_value,current_lr_value=\\\n",
    "      sess.run(\n",
    "     [train_step,cross_entropy,l2_loss,total_loss,\n",
    "      current_learning_rate],\n",
    "     feed_dict={x:batch_xs,y:batch_ys,\n",
    "       init_learning_rate:lr})\n",
    "        \n",
    "    if(step+1)%100==0:\n",
    "        print('step %d,entropy loss: %f,l2_loss: %f,total loss: %f'%\n",
    "        (step+1,loss,l2_loss_value,total_loss_value))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,\n",
    "        y_:mnist.test.labels}))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
